{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Chapter 19 using LM with Penalty:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "using VMLS\n",
    "using LinearAlgebra"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "levenberg_marquardt (generic function with 1 method)"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function levenberg_marquardt(f,Df,x1, lambda1; kmax=100, tol=1e-6)\n",
    "    n = length(x1)\n",
    "    x = x1\n",
    "    lambda = lambda1\n",
    "    objectives = zeros(0,1)\n",
    "    residuals = zeros(0,1)\n",
    "    for k = 1:kmax\n",
    "        fk = f(x)\n",
    "        Dfk = Df(x)\n",
    "        objectives = [objectives; norm(fk)^2]\n",
    "        residuals = [residuals; norm(2*Dfk'*fk)]\n",
    "        if norm(2*Dfk'*fk)<tol\n",
    "            break\n",
    "        end;\n",
    "        xt = x - [Dfk; sqrt(lambda)*eye(n)] \\ [fk; zeros(n)]\n",
    "        if norm(f(xt)) < norm(fk)\n",
    "            lambda = .8*lambda\n",
    "            x = xt\n",
    "        else\n",
    "            lambda = 2.0 * lambda\n",
    "        end\n",
    "    end\n",
    "    return x,Dict([(\"objectives\", objectives), (\"residuals\", residuals)])\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "penalty_method (generic function with 1 method)"
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function penalty_method(f,Df, g,Dg, x1, lambda1; kmax = 100, feas_tol = 1e-4, oc_tol = 1e-4)\n",
    "    x = x1\n",
    "    mu = 1.0\n",
    "    feas_res = [norm(g(x))]\n",
    "    oc_res = [norm(2*Df(x)'*f(x)+2*mu*Dg(x)'*g(x))]\n",
    "    lm_iters = zeros(Int64, 0, 1);\n",
    "    F(x) = [f(x); sqrt(mu)*g(x)]\n",
    "    DF(x) = [Df(x);sqrt(mu)*Dg(x)]\n",
    "    for k = 1:kmax\n",
    "        x,hist = levenberg_marquardt(F,DF,x,lambda1, tol=oc_tol)\n",
    "        feas_res = [feas_res;norm(g(x))]\n",
    "        oc_res = [oc_res; hist[\"residuals\"][end]]\n",
    "        lm_iters = [lm_iters; length(hist[\"residuals\"])]\n",
    "        if norm(g(x)) < feas_tol\n",
    "            break\n",
    "        end\n",
    "        mu = 2*mu\n",
    "    end\n",
    "    return x, Dict([(\"lm_iterations\", lm_iters), (\"feas_res\", feas_res), (\"oc_res\", oc_res)])\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "f(x) = [x[1] + exp(-x[2]); x[1]^2 + 2*x[2] + 1];\n",
    "Df(x) = [1.0 -exp(-x[2]); 2*x[1] 2];\n",
    "g(x) = [x[1] + x[1]^3 + x[2] + x[2]^2];\n",
    "Dg(x) = [1.0 + 3*x[1]^2 1+2*x[2]];\n",
    "x,hist = penalty_method(f,Df,g,Dg,[0.5,-0.5],1.0);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "using Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"600\" height=\"400\" viewBox=\"0 0 2400 1600\">\n",
       "<defs>\n",
       "  <clipPath id=\"clip9800\">\n",
       "    <rect x=\"0\" y=\"0\" width=\"2400\" height=\"1600\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<polygon clip-path=\"url(#clip9800)\" points=\"\n",
       "0,1600 2400,1600 2400,0 0,0 \n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<defs>\n",
       "  <clipPath id=\"clip9801\">\n",
       "    <rect x=\"480\" y=\"0\" width=\"1681\" height=\"1600\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<polygon clip-path=\"url(#clip9800)\" points=\"\n",
       "279.82,1425.62 2352.76,1425.62 2352.76,47.2441 279.82,47.2441 \n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<defs>\n",
       "  <clipPath id=\"clip9802\">\n",
       "    <rect x=\"279\" y=\"47\" width=\"2074\" height=\"1379\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<polyline clip-path=\"url(#clip9802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  338.488,1425.62 338.488,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip9802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  628.207,1425.62 628.207,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip9802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  917.925,1425.62 917.925,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip9802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  1207.64,1425.62 1207.64,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip9802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  1497.36,1425.62 1497.36,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip9802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  1787.08,1425.62 1787.08,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip9802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  2076.8,1425.62 2076.8,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip9802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  279.82,1275.62 2352.76,1275.62 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip9802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  279.82,1075.24 2352.76,1075.24 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip9802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  279.82,874.866 2352.76,874.866 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip9802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  279.82,674.491 2352.76,674.491 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip9802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  279.82,474.116 2352.76,474.116 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip9802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  279.82,273.741 2352.76,273.741 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip9802)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  279.82,73.3657 2352.76,73.3657 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip9800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  279.82,1425.62 2352.76,1425.62 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip9800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  279.82,1425.62 279.82,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip9800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  338.488,1425.62 338.488,1404.94 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip9800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  628.207,1425.62 628.207,1404.94 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip9800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  917.925,1425.62 917.925,1404.94 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip9800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1207.64,1425.62 1207.64,1404.94 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip9800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1497.36,1425.62 1497.36,1404.94 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip9800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1787.08,1425.62 1787.08,1404.94 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip9800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  2076.8,1425.62 2076.8,1404.94 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip9800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  279.82,1275.62 310.914,1275.62 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip9800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  279.82,1075.24 310.914,1075.24 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip9800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  279.82,874.866 310.914,874.866 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip9800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  279.82,674.491 310.914,674.491 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip9800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  279.82,474.116 310.914,474.116 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip9800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  279.82,273.741 310.914,273.741 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip9800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  279.82,73.3657 310.914,73.3657 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip9800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 338.488, 1479.62)\" x=\"338.488\" y=\"1479.62\">0</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip9800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 628.207, 1479.62)\" x=\"628.207\" y=\"1479.62\">20</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip9800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 917.925, 1479.62)\" x=\"917.925\" y=\"1479.62\">40</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip9800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 1207.64, 1479.62)\" x=\"1207.64\" y=\"1479.62\">60</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip9800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 1497.36, 1479.62)\" x=\"1497.36\" y=\"1479.62\">80</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip9800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 1787.08, 1479.62)\" x=\"1787.08\" y=\"1479.62\">100</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip9800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 2076.8, 1479.62)\" x=\"2076.8\" y=\"1479.62\">120</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip9800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 146.839, 1299.34)\" x=\"146.839\" y=\"1299.34\">10</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip9800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:38px; text-anchor:start;\" transform=\"rotate(0, 200.364, 1271.93)\" x=\"200.364\" y=\"1271.93\">-</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip9800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:38px; text-anchor:start;\" transform=\"rotate(0, 223.203, 1271.93)\" x=\"223.203\" y=\"1271.93\">5 </text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip9800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 146.839, 1098.97)\" x=\"146.839\" y=\"1098.97\">10</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip9800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:38px; text-anchor:start;\" transform=\"rotate(0, 200.364, 1071.56)\" x=\"200.364\" y=\"1071.56\">-</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip9800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:38px; text-anchor:start;\" transform=\"rotate(0, 223.203, 1071.56)\" x=\"223.203\" y=\"1071.56\">4 </text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip9800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 146.839, 898.594)\" x=\"146.839\" y=\"898.594\">10</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip9800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:38px; text-anchor:start;\" transform=\"rotate(0, 200.364, 871.183)\" x=\"200.364\" y=\"871.183\">-</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip9800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:38px; text-anchor:start;\" transform=\"rotate(0, 223.203, 871.183)\" x=\"223.203\" y=\"871.183\">3 </text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip9800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 146.839, 698.219)\" x=\"146.839\" y=\"698.219\">10</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip9800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:38px; text-anchor:start;\" transform=\"rotate(0, 200.364, 670.808)\" x=\"200.364\" y=\"670.808\">-</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip9800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:38px; text-anchor:start;\" transform=\"rotate(0, 223.203, 670.808)\" x=\"223.203\" y=\"670.808\">2 </text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip9800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 146.839, 497.844)\" x=\"146.839\" y=\"497.844\">10</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip9800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:38px; text-anchor:start;\" transform=\"rotate(0, 200.364, 470.433)\" x=\"200.364\" y=\"470.433\">-</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip9800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:38px; text-anchor:start;\" transform=\"rotate(0, 223.203, 470.433)\" x=\"223.203\" y=\"470.433\">1 </text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip9800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 169.679, 297.468)\" x=\"169.679\" y=\"297.468\">10</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip9800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:38px; text-anchor:start;\" transform=\"rotate(0, 223.203, 270.058)\" x=\"223.203\" y=\"270.058\">0 </text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip9800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 169.679, 97.0933)\" x=\"169.679\" y=\"97.0933\">10</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip9800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:38px; text-anchor:start;\" transform=\"rotate(0, 223.203, 69.6829)\" x=\"223.203\" y=\"69.6829\">1 </text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip9800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:66px; text-anchor:middle;\" transform=\"rotate(0, 1316.29, 1559.48)\" x=\"1316.29\" y=\"1559.48\">Cumulative Levenberg--Marquardt iterations</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip9800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:66px; text-anchor:middle;\" transform=\"rotate(-90, 89.2861, 736.431)\" x=\"89.2861\" y=\"736.431\">Residual</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip9802)\" style=\"stroke:#009af9; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  338.488,359.094 1164.19,359.094 1164.19,343.892 1352.5,343.892 1352.5,377.041 1482.88,377.041 1482.88,418.497 1584.28,418.497 1584.28,467.433 1671.19,467.433 \n",
       "  1671.19,521.538 1758.11,521.538 1758.11,578.63 1830.54,578.63 1830.54,637.308 1902.97,637.308 1902.97,696.802 1975.4,696.802 1975.4,756.707 2033.34,756.707 \n",
       "  2033.34,816.817 2091.28,816.817 2091.28,877.033 2149.23,877.033 2149.23,937.3 2207.17,937.3 2207.17,997.593 2250.63,997.593 2250.63,1057.9 2294.09,1057.9 \n",
       "  2294.09,1118.21 \n",
       "  \"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"338.488\" cy=\"359.094\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"338.488\" cy=\"359.094\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1164.19\" cy=\"359.094\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"1164.19\" cy=\"359.094\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1164.19\" cy=\"343.892\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"1164.19\" cy=\"343.892\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1352.5\" cy=\"343.892\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"1352.5\" cy=\"343.892\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1352.5\" cy=\"377.041\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"1352.5\" cy=\"377.041\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1482.88\" cy=\"377.041\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"1482.88\" cy=\"377.041\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1482.88\" cy=\"418.497\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"1482.88\" cy=\"418.497\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1584.28\" cy=\"418.497\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"1584.28\" cy=\"418.497\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1584.28\" cy=\"467.433\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"1584.28\" cy=\"467.433\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1671.19\" cy=\"467.433\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"1671.19\" cy=\"467.433\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1671.19\" cy=\"521.538\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"1671.19\" cy=\"521.538\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1758.11\" cy=\"521.538\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"1758.11\" cy=\"521.538\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1758.11\" cy=\"578.63\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"1758.11\" cy=\"578.63\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1830.54\" cy=\"578.63\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"1830.54\" cy=\"578.63\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1830.54\" cy=\"637.308\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"1830.54\" cy=\"637.308\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1902.97\" cy=\"637.308\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"1902.97\" cy=\"637.308\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1902.97\" cy=\"696.802\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"1902.97\" cy=\"696.802\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1975.4\" cy=\"696.802\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"1975.4\" cy=\"696.802\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1975.4\" cy=\"756.707\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"1975.4\" cy=\"756.707\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2033.34\" cy=\"756.707\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"2033.34\" cy=\"756.707\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2033.34\" cy=\"816.817\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"2033.34\" cy=\"816.817\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2091.28\" cy=\"816.817\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"2091.28\" cy=\"816.817\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2091.28\" cy=\"877.033\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"2091.28\" cy=\"877.033\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2149.23\" cy=\"877.033\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"2149.23\" cy=\"877.033\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2149.23\" cy=\"937.3\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"2149.23\" cy=\"937.3\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2207.17\" cy=\"937.3\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"2207.17\" cy=\"937.3\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2207.17\" cy=\"997.593\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"2207.17\" cy=\"997.593\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2250.63\" cy=\"997.593\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"2250.63\" cy=\"997.593\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2250.63\" cy=\"1057.9\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"2250.63\" cy=\"1057.9\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2294.09\" cy=\"1057.9\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"2294.09\" cy=\"1057.9\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2294.09\" cy=\"1118.21\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"2294.09\" cy=\"1118.21\" r=\"14\"/>\n",
       "<polyline clip-path=\"url(#clip9802)\" style=\"stroke:#e26f46; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  338.488,86.2547 1164.19,86.2547 1164.19,1082.06 1352.5,1082.06 1352.5,1096.03 1482.88,1096.03 1482.88,1108.08 1584.28,1108.08 1584.28,1119.97 1671.19,1119.97 \n",
       "  1671.19,1202.43 1758.11,1202.43 1758.11,1386.61 1830.54,1386.61 1830.54,1162.24 1902.97,1162.24 1902.97,1247.79 1975.4,1247.79 1975.4,1324.58 2033.34,1324.58 \n",
       "  2033.34,1106.06 2091.28,1106.06 2091.28,1164.95 2149.23,1164.95 2149.23,1224.58 2207.17,1224.58 2207.17,1284.56 2250.63,1284.56 2250.63,1112.31 2294.09,1112.31 \n",
       "  2294.09,1171.66 \n",
       "  \"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"338.488\" cy=\"86.2547\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"338.488\" cy=\"86.2547\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1164.19\" cy=\"86.2547\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"1164.19\" cy=\"86.2547\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1164.19\" cy=\"1082.06\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"1164.19\" cy=\"1082.06\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1352.5\" cy=\"1082.06\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"1352.5\" cy=\"1082.06\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1352.5\" cy=\"1096.03\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"1352.5\" cy=\"1096.03\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1482.88\" cy=\"1096.03\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"1482.88\" cy=\"1096.03\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1482.88\" cy=\"1108.08\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"1482.88\" cy=\"1108.08\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1584.28\" cy=\"1108.08\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"1584.28\" cy=\"1108.08\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1584.28\" cy=\"1119.97\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"1584.28\" cy=\"1119.97\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1671.19\" cy=\"1119.97\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"1671.19\" cy=\"1119.97\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1671.19\" cy=\"1202.43\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"1671.19\" cy=\"1202.43\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1758.11\" cy=\"1202.43\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"1758.11\" cy=\"1202.43\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1758.11\" cy=\"1386.61\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"1758.11\" cy=\"1386.61\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1830.54\" cy=\"1386.61\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"1830.54\" cy=\"1386.61\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1830.54\" cy=\"1162.24\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"1830.54\" cy=\"1162.24\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1902.97\" cy=\"1162.24\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"1902.97\" cy=\"1162.24\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1902.97\" cy=\"1247.79\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"1902.97\" cy=\"1247.79\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1975.4\" cy=\"1247.79\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"1975.4\" cy=\"1247.79\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1975.4\" cy=\"1324.58\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"1975.4\" cy=\"1324.58\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2033.34\" cy=\"1324.58\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"2033.34\" cy=\"1324.58\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2033.34\" cy=\"1106.06\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"2033.34\" cy=\"1106.06\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2091.28\" cy=\"1106.06\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"2091.28\" cy=\"1106.06\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2091.28\" cy=\"1164.95\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"2091.28\" cy=\"1164.95\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2149.23\" cy=\"1164.95\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"2149.23\" cy=\"1164.95\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2149.23\" cy=\"1224.58\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"2149.23\" cy=\"1224.58\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2207.17\" cy=\"1224.58\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"2207.17\" cy=\"1224.58\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2207.17\" cy=\"1284.56\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"2207.17\" cy=\"1284.56\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2250.63\" cy=\"1284.56\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"2250.63\" cy=\"1284.56\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2250.63\" cy=\"1112.31\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"2250.63\" cy=\"1112.31\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2294.09\" cy=\"1112.31\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"2294.09\" cy=\"1112.31\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2294.09\" cy=\"1171.66\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip9802)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"2294.09\" cy=\"1171.66\" r=\"14\"/>\n",
       "<polygon clip-path=\"url(#clip9800)\" points=\"\n",
       "1818.67,312.204 2280.76,312.204 2280.76,130.764 1818.67,130.764 \n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip9800)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1818.67,312.204 2280.76,312.204 2280.76,130.764 1818.67,130.764 1818.67,312.204 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip9800)\" style=\"stroke:#009af9; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1842.67,191.244 1986.67,191.244 \n",
       "  \"/>\n",
       "<circle clip-path=\"url(#clip9800)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1926.67\" cy=\"191.244\" r=\"25\"/>\n",
       "<circle clip-path=\"url(#clip9800)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"1926.67\" cy=\"191.244\" r=\"21\"/>\n",
       "<g clip-path=\"url(#clip9800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 2010.67, 208.744)\" x=\"2010.67\" y=\"208.744\">Feasibility</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip9800)\" style=\"stroke:#e26f46; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1842.67,251.724 1986.67,251.724 \n",
       "  \"/>\n",
       "<circle clip-path=\"url(#clip9800)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1926.67\" cy=\"251.724\" r=\"25\"/>\n",
       "<circle clip-path=\"url(#clip9800)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"1926.67\" cy=\"251.724\" r=\"21\"/>\n",
       "<g clip-path=\"url(#clip9800)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 2010.67, 269.224)\" x=\"2010.67\" y=\"269.224\">Opt. cond.</text>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": 176,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cum_lm_iters = cumsum(hist[\"lm_iterations\"], dims=1);\n",
    "itr = vcat([0], [[i;i] for i in cum_lm_iters]...)\n",
    "feas_res = vcat([\n",
    "    [r;r] for r in hist[\"feas_res\"][1:end-1]]...,\n",
    "    hist[\"feas_res\"][end]);\n",
    "oc_res = vcat([\n",
    "        [r;r] for r in hist[\"oc_res\"][1:end-1]]...,\n",
    "    hist[\"oc_res\"][end]);\n",
    "plot(itr,feas_res,shape=:circle,label = \"Feasibility\")\n",
    "plot!(itr,oc_res, shape=:circle, label = \"Opt. cond.\")\n",
    "plot!(yscale = :log10,\n",
    "xlabel = \"Cumulative Levenberg--Marquardt iterations\",\n",
    "ylabel = \"Residual\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Augmented Lagrangian"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 302,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "levenberg_marquardt (generic function with 1 method)"
      ]
     },
     "execution_count": 302,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function levenberg_marquardt(f,Df,x1, lambda1; kmax=100, tol=1e-6)\n",
    "    n = length(x1)\n",
    "    x = x1\n",
    "    lambda = lambda1\n",
    "    objectives = zeros(0,1)\n",
    "    residuals = zeros(0,1)\n",
    "    for k = 1:kmax\n",
    "        fk = f(x)\n",
    "        Dfk = Df(x)\n",
    "        objectives = [objectives; norm(fk)^2]\n",
    "        residuals = [residuals; norm(2*Dfk'*fk)]\n",
    "        if norm(2*Dfk'*fk)<tol\n",
    "            break\n",
    "        end;\n",
    "        xt = x - [Dfk; sqrt(lambda)*eye(n)] \\ [fk; zeros(n)]\n",
    "        if norm(f(xt)) < norm(fk)\n",
    "            lambda = .8*lambda\n",
    "            x = xt\n",
    "        else\n",
    "            lambda = 2.0 * lambda\n",
    "        end\n",
    "    end\n",
    "    return x,Dict([(\"objectives\", objectives), (\"residuals\", residuals)])\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 356,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "aug_lag_method (generic function with 1 method)"
      ]
     },
     "execution_count": 356,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function aug_lag_method(f,Df,g,Dg,x1,lambda1; kmax = 100, feas_tol = 1e-4, oc_tol = 1e-4)\n",
    "    x = x1\n",
    "    z = zeros(length(g(x)))\n",
    "    mu = 1.0\n",
    "    feas_res = [norm(g(x))]\n",
    "    oc_res = [norm(2*Df(x)'*f(x)+2*mu*Dg(x)'*z)]\n",
    "    lm_iters = zeros(Int64, 0, 1);\n",
    "    F(x) = [f(x); sqrt(mu)*(g(x)+z/(2*mu))]\n",
    "    DF(x) = [Df(x);sqrt(mu)*Dg(x)]\n",
    "    for k = 1:kmax\n",
    "        x,hist = levenberg_marquardt(F,DF,x,lambda1, tol=oc_tol)\n",
    "        z = z+2*mu*g(x)\n",
    "        feas_res = [feas_res;norm(g(x))]\n",
    "        oc_res = [oc_res; hist[\"residuals\"][end]]\n",
    "        lm_iters = [lm_iters; length(hist[\"residuals\"])]\n",
    "        if norm(g(x)) < feas_tol\n",
    "            break\n",
    "        end\n",
    "        mu = norm(g(x))<.25*feas_res[end-1] ? mu : mu*2\n",
    "    end\n",
    "    return x, z, Dict([(\"lm_iterations\", lm_iters), (\"feas_res\", feas_res), (\"oc_res\", oc_res)])\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 357,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([-1.86466e-5, -1.50086e-5], [-1.99996])"
      ]
     },
     "execution_count": 357,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x,z,hist = aug_lag_method(f,Df,g,Dg,[0.5,-0.5],1.0);\n",
    "x,z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 355,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"600\" height=\"400\" viewBox=\"0 0 2400 1600\">\n",
       "<defs>\n",
       "  <clipPath id=\"clip5400\">\n",
       "    <rect x=\"0\" y=\"0\" width=\"2400\" height=\"1600\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<polygon clip-path=\"url(#clip5400)\" points=\"\n",
       "0,1600 2400,1600 2400,0 0,0 \n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<defs>\n",
       "  <clipPath id=\"clip5401\">\n",
       "    <rect x=\"480\" y=\"0\" width=\"1681\" height=\"1600\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<polygon clip-path=\"url(#clip5400)\" points=\"\n",
       "279.82,1425.62 2352.76,1425.62 2352.76,47.2441 279.82,47.2441 \n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<defs>\n",
       "  <clipPath id=\"clip5402\">\n",
       "    <rect x=\"279\" y=\"47\" width=\"2074\" height=\"1379\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<polyline clip-path=\"url(#clip5402)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  338.488,1425.62 338.488,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip5402)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  820.757,1425.62 820.757,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip5402)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  1303.03,1425.62 1303.03,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip5402)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  1785.29,1425.62 1785.29,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip5402)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  2267.56,1425.62 2267.56,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip5402)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  279.82,1307.75 2352.76,1307.75 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip5402)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  279.82,1057.92 2352.76,1057.92 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip5402)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  279.82,808.085 2352.76,808.085 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip5402)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  279.82,558.251 2352.76,558.251 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip5402)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  279.82,308.417 2352.76,308.417 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip5402)\" style=\"stroke:#000000; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  279.82,58.5838 2352.76,58.5838 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip5400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  279.82,1425.62 2352.76,1425.62 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip5400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  279.82,1425.62 279.82,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip5400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  338.488,1425.62 338.488,1404.94 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip5400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  820.757,1425.62 820.757,1404.94 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip5400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1303.03,1425.62 1303.03,1404.94 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip5400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1785.29,1425.62 1785.29,1404.94 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip5400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  2267.56,1425.62 2267.56,1404.94 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip5400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  279.82,1307.75 310.914,1307.75 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip5400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  279.82,1057.92 310.914,1057.92 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip5400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  279.82,808.085 310.914,808.085 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip5400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  279.82,558.251 310.914,558.251 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip5400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  279.82,308.417 310.914,308.417 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip5400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  279.82,58.5838 310.914,58.5838 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip5400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 338.488, 1479.62)\" x=\"338.488\" y=\"1479.62\">0</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip5400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 820.757, 1479.62)\" x=\"820.757\" y=\"1479.62\">200</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip5400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 1303.03, 1479.62)\" x=\"1303.03\" y=\"1479.62\">400</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip5400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 1785.29, 1479.62)\" x=\"1785.29\" y=\"1479.62\">600</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip5400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:middle;\" transform=\"rotate(0, 2267.56, 1479.62)\" x=\"2267.56\" y=\"1479.62\">800</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip5400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 146.839, 1331.48)\" x=\"146.839\" y=\"1331.48\">10</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip5400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:38px; text-anchor:start;\" transform=\"rotate(0, 200.364, 1304.07)\" x=\"200.364\" y=\"1304.07\">-</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip5400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:38px; text-anchor:start;\" transform=\"rotate(0, 223.203, 1304.07)\" x=\"223.203\" y=\"1304.07\">4 </text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip5400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 146.839, 1081.65)\" x=\"146.839\" y=\"1081.65\">10</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip5400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:38px; text-anchor:start;\" transform=\"rotate(0, 200.364, 1054.24)\" x=\"200.364\" y=\"1054.24\">-</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip5400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:38px; text-anchor:start;\" transform=\"rotate(0, 223.203, 1054.24)\" x=\"223.203\" y=\"1054.24\">3 </text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip5400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 146.839, 831.812)\" x=\"146.839\" y=\"831.812\">10</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip5400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:38px; text-anchor:start;\" transform=\"rotate(0, 200.364, 804.402)\" x=\"200.364\" y=\"804.402\">-</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip5400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:38px; text-anchor:start;\" transform=\"rotate(0, 223.203, 804.402)\" x=\"223.203\" y=\"804.402\">2 </text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip5400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 146.839, 581.979)\" x=\"146.839\" y=\"581.979\">10</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip5400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:38px; text-anchor:start;\" transform=\"rotate(0, 200.364, 554.568)\" x=\"200.364\" y=\"554.568\">-</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip5400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:38px; text-anchor:start;\" transform=\"rotate(0, 223.203, 554.568)\" x=\"223.203\" y=\"554.568\">1 </text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip5400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 169.679, 332.145)\" x=\"169.679\" y=\"332.145\">10</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip5400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:38px; text-anchor:start;\" transform=\"rotate(0, 223.203, 304.735)\" x=\"223.203\" y=\"304.735\">0 </text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip5400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 169.679, 82.3114)\" x=\"169.679\" y=\"82.3114\">10</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip5400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:38px; text-anchor:start;\" transform=\"rotate(0, 223.203, 54.901)\" x=\"223.203\" y=\"54.901\">1 </text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip5400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:66px; text-anchor:middle;\" transform=\"rotate(0, 1316.29, 1559.48)\" x=\"1316.29\" y=\"1559.48\">Cumulative Levenberg--Marquardt iterations</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip5400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:66px; text-anchor:middle;\" transform=\"rotate(-90, 89.2861, 736.431)\" x=\"89.2861\" y=\"736.431\">Residual</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip5402)\" style=\"stroke:#009af9; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  338.488,414.839 475.935,414.839 475.935,395.884 717.069,395.884 717.069,461.846 958.203,461.846 958.203,524.066 1199.34,524.066 1199.34,586.904 1440.47,586.904 \n",
       "  1440.47,649.913 1681.61,649.913 1681.61,713.015 1922.74,713.015 1922.74,777.485 2163.88,777.485 2163.88,842.826 2204.87,842.826 2204.87,906.645 2228.98,906.645 \n",
       "  2228.98,972.566 2245.86,972.566 2245.86,1035.36 2260.33,1035.36 2260.33,1103.49 2269.97,1103.49 2269.97,1168.34 2279.62,1168.34 2279.62,1230.74 2286.85,1230.74 \n",
       "  2286.85,1282.77 2294.09,1282.77 2294.09,1350.9 \n",
       "  \"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"338.488\" cy=\"414.839\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"338.488\" cy=\"414.839\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"475.935\" cy=\"414.839\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"475.935\" cy=\"414.839\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"475.935\" cy=\"395.884\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"475.935\" cy=\"395.884\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"717.069\" cy=\"395.884\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"717.069\" cy=\"395.884\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"717.069\" cy=\"461.846\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"717.069\" cy=\"461.846\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"958.203\" cy=\"461.846\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"958.203\" cy=\"461.846\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"958.203\" cy=\"524.066\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"958.203\" cy=\"524.066\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1199.34\" cy=\"524.066\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"1199.34\" cy=\"524.066\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1199.34\" cy=\"586.904\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"1199.34\" cy=\"586.904\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1440.47\" cy=\"586.904\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"1440.47\" cy=\"586.904\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1440.47\" cy=\"649.913\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"1440.47\" cy=\"649.913\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1681.61\" cy=\"649.913\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"1681.61\" cy=\"649.913\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1681.61\" cy=\"713.015\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"1681.61\" cy=\"713.015\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1922.74\" cy=\"713.015\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"1922.74\" cy=\"713.015\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1922.74\" cy=\"777.485\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"1922.74\" cy=\"777.485\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2163.88\" cy=\"777.485\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"2163.88\" cy=\"777.485\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2163.88\" cy=\"842.826\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"2163.88\" cy=\"842.826\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2204.87\" cy=\"842.826\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"2204.87\" cy=\"842.826\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2204.87\" cy=\"906.645\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"2204.87\" cy=\"906.645\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2228.98\" cy=\"906.645\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"2228.98\" cy=\"906.645\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2228.98\" cy=\"972.566\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"2228.98\" cy=\"972.566\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2245.86\" cy=\"972.566\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"2245.86\" cy=\"972.566\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2245.86\" cy=\"1035.36\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"2245.86\" cy=\"1035.36\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2260.33\" cy=\"1035.36\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"2260.33\" cy=\"1035.36\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2260.33\" cy=\"1103.49\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"2260.33\" cy=\"1103.49\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2269.97\" cy=\"1103.49\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"2269.97\" cy=\"1103.49\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2269.97\" cy=\"1168.34\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"2269.97\" cy=\"1168.34\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2279.62\" cy=\"1168.34\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"2279.62\" cy=\"1168.34\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2279.62\" cy=\"1230.74\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"2279.62\" cy=\"1230.74\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2286.85\" cy=\"1230.74\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"2286.85\" cy=\"1230.74\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2286.85\" cy=\"1282.77\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"2286.85\" cy=\"1282.77\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2294.09\" cy=\"1282.77\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"2294.09\" cy=\"1282.77\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2294.09\" cy=\"1350.9\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"2294.09\" cy=\"1350.9\" r=\"14\"/>\n",
       "<polyline clip-path=\"url(#clip5402)\" style=\"stroke:#e26f46; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  338.488,86.2547 475.935,86.2547 475.935,1316.26 717.069,1316.26 717.069,906.18 958.203,906.18 958.203,986.99 1199.34,986.99 1199.34,1039.63 1440.47,1039.63 \n",
       "  1440.47,1082.35 1681.61,1082.35 1681.61,1135.92 1922.74,1135.92 1922.74,1195.28 2163.88,1195.28 2163.88,1259.4 2204.87,1259.4 2204.87,1309.91 2228.98,1309.91 \n",
       "  2228.98,1318.16 2245.86,1318.16 2245.86,1319.44 2260.33,1319.44 2260.33,1340.69 2269.97,1340.69 2269.97,1313.68 2279.62,1313.68 2279.62,1386.12 2286.85,1386.12 \n",
       "  2286.85,1362.26 2294.09,1362.26 2294.09,1386.61 \n",
       "  \"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"338.488\" cy=\"86.2547\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"338.488\" cy=\"86.2547\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"475.935\" cy=\"86.2547\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"475.935\" cy=\"86.2547\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"475.935\" cy=\"1316.26\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"475.935\" cy=\"1316.26\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"717.069\" cy=\"1316.26\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"717.069\" cy=\"1316.26\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"717.069\" cy=\"906.18\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"717.069\" cy=\"906.18\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"958.203\" cy=\"906.18\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"958.203\" cy=\"906.18\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"958.203\" cy=\"986.99\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"958.203\" cy=\"986.99\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1199.34\" cy=\"986.99\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"1199.34\" cy=\"986.99\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1199.34\" cy=\"1039.63\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"1199.34\" cy=\"1039.63\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1440.47\" cy=\"1039.63\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"1440.47\" cy=\"1039.63\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1440.47\" cy=\"1082.35\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"1440.47\" cy=\"1082.35\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1681.61\" cy=\"1082.35\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"1681.61\" cy=\"1082.35\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1681.61\" cy=\"1135.92\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"1681.61\" cy=\"1135.92\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1922.74\" cy=\"1135.92\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"1922.74\" cy=\"1135.92\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1922.74\" cy=\"1195.28\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"1922.74\" cy=\"1195.28\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2163.88\" cy=\"1195.28\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"2163.88\" cy=\"1195.28\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2163.88\" cy=\"1259.4\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"2163.88\" cy=\"1259.4\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2204.87\" cy=\"1259.4\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"2204.87\" cy=\"1259.4\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2204.87\" cy=\"1309.91\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"2204.87\" cy=\"1309.91\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2228.98\" cy=\"1309.91\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"2228.98\" cy=\"1309.91\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2228.98\" cy=\"1318.16\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"2228.98\" cy=\"1318.16\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2245.86\" cy=\"1318.16\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"2245.86\" cy=\"1318.16\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2245.86\" cy=\"1319.44\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"2245.86\" cy=\"1319.44\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2260.33\" cy=\"1319.44\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"2260.33\" cy=\"1319.44\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2260.33\" cy=\"1340.69\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"2260.33\" cy=\"1340.69\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2269.97\" cy=\"1340.69\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"2269.97\" cy=\"1340.69\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2269.97\" cy=\"1313.68\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"2269.97\" cy=\"1313.68\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2279.62\" cy=\"1313.68\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"2279.62\" cy=\"1313.68\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2279.62\" cy=\"1386.12\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"2279.62\" cy=\"1386.12\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2286.85\" cy=\"1386.12\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"2286.85\" cy=\"1386.12\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2286.85\" cy=\"1362.26\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"2286.85\" cy=\"1362.26\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2294.09\" cy=\"1362.26\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"2294.09\" cy=\"1362.26\" r=\"14\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"2294.09\" cy=\"1386.61\" r=\"18\"/>\n",
       "<circle clip-path=\"url(#clip5402)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"2294.09\" cy=\"1386.61\" r=\"14\"/>\n",
       "<polygon clip-path=\"url(#clip5400)\" points=\"\n",
       "1818.67,312.204 2280.76,312.204 2280.76,130.764 1818.67,130.764 \n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip5400)\" style=\"stroke:#000000; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1818.67,312.204 2280.76,312.204 2280.76,130.764 1818.67,130.764 1818.67,312.204 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip5400)\" style=\"stroke:#009af9; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1842.67,191.244 1986.67,191.244 \n",
       "  \"/>\n",
       "<circle clip-path=\"url(#clip5400)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1926.67\" cy=\"191.244\" r=\"25\"/>\n",
       "<circle clip-path=\"url(#clip5400)\" style=\"fill:#009af9; stroke:none; fill-opacity:1\" cx=\"1926.67\" cy=\"191.244\" r=\"21\"/>\n",
       "<g clip-path=\"url(#clip5400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 2010.67, 208.744)\" x=\"2010.67\" y=\"208.744\">Feasibility</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip5400)\" style=\"stroke:#e26f46; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1842.67,251.724 1986.67,251.724 \n",
       "  \"/>\n",
       "<circle clip-path=\"url(#clip5400)\" style=\"fill:#000000; stroke:none; fill-opacity:1\" cx=\"1926.67\" cy=\"251.724\" r=\"25\"/>\n",
       "<circle clip-path=\"url(#clip5400)\" style=\"fill:#e26f46; stroke:none; fill-opacity:1\" cx=\"1926.67\" cy=\"251.724\" r=\"21\"/>\n",
       "<g clip-path=\"url(#clip5400)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:48px; text-anchor:start;\" transform=\"rotate(0, 2010.67, 269.224)\" x=\"2010.67\" y=\"269.224\">Opt. cond.</text>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": 355,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cum_lm_iters = cumsum(hist[\"lm_iterations\"], dims=1);\n",
    "itr = vcat([0], [[i;i] for i in cum_lm_iters]...)\n",
    "feas_res = vcat([\n",
    "    [r;r] for r in hist[\"feas_res\"][1:end-1]]...,\n",
    "    hist[\"feas_res\"][end]);\n",
    "oc_res = vcat([\n",
    "        [r;r] for r in hist[\"oc_res\"][1:end-1]]...,\n",
    "    hist[\"oc_res\"][end]);\n",
    "plot(itr,feas_res,shape=:circle,label = \"Feasibility\")\n",
    "plot!(itr,oc_res, shape=:circle, label = \"Opt. cond.\")\n",
    "plot!(yscale = :log10,\n",
    "xlabel = \"Cumulative Levenberg--Marquardt iterations\",\n",
    "ylabel = \"Residual\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 1.1.1",
   "language": "julia",
   "name": "julia-1.1"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.1.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
